1 package org.apache.lucene.uninverting;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 import java.io.ByteArrayOutputStream;
20 import java.io.IOException;
21 import java.io.PrintStream;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.LinkedHashSet;
25 import java.util.List;
26 import java.util.concurrent.CyclicBarrier;
27 import java.util.concurrent.atomic.AtomicBoolean;
28 import java.util.concurrent.atomic.AtomicInteger;
29
30 import org.apache.lucene.analysis.MockAnalyzer;
31 import org.apache.lucene.document.BinaryDocValuesField;
32 import org.apache.lucene.document.Document;
33 import org.apache.lucene.document.DoubleField;
34 import org.apache.lucene.document.Field;
35 import org.apache.lucene.document.Field.Store;
36 import org.apache.lucene.document.FloatField;
37 import org.apache.lucene.document.IntField;
38 import org.apache.lucene.document.LongField;
39 import org.apache.lucene.document.NumericDocValuesField;
40 import org.apache.lucene.document.SortedDocValuesField;
41 import org.apache.lucene.document.SortedSetDocValuesField;
42 import org.apache.lucene.document.StoredField;
43 import org.apache.lucene.index.LeafReader;
44 import org.apache.lucene.index.BinaryDocValues;
45 import org.apache.lucene.index.DirectoryReader;
46 import org.apache.lucene.index.IndexReader;
47 import org.apache.lucene.index.IndexWriter;
48 import org.apache.lucene.index.IndexWriterConfig;
49 import org.apache.lucene.index.NumericDocValues;
50 import org.apache.lucene.index.RandomIndexWriter;
51 import org.apache.lucene.index.SlowCompositeReaderWrapper;
52 import org.apache.lucene.index.SortedDocValues;
53 import org.apache.lucene.index.SortedSetDocValues;
54 import org.apache.lucene.index.Terms;
55 import org.apache.lucene.index.TermsEnum;
56 import org.apache.lucene.store.Directory;
57 import org.apache.lucene.util.Bits;
58 import org.apache.lucene.util.BytesRef;
59 import org.apache.lucene.util.IOUtils;
60 import org.apache.lucene.util.LuceneTestCase;
61 import org.apache.lucene.util.NumericUtils;
62 import org.apache.lucene.util.TestUtil;
63 import org.junit.AfterClass;
64 import org.junit.BeforeClass;
65
66 public class TestFieldCache extends LuceneTestCase {
67 private static LeafReader reader;
68 private static int NUM_DOCS;
69 private static int NUM_ORDS;
70 private static String[] unicodeStrings;
71 private static BytesRef[][] multiValued;
72 private static Directory directory;
73
74 @BeforeClass
75 public static void beforeClass() throws Exception {
76 NUM_DOCS = atLeast(500);
77 NUM_ORDS = atLeast(2);
78 directory = newDirectory();
79 RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
80 long theLong = Long.MAX_VALUE;
81 double theDouble = Double.MAX_VALUE;
82 int theInt = Integer.MAX_VALUE;
83 float theFloat = Float.MAX_VALUE;
84 unicodeStrings = new String[NUM_DOCS];
85 multiValued = new BytesRef[NUM_DOCS][NUM_ORDS];
86 if (VERBOSE) {
87 System.out.println("TEST: setUp");
88 }
89 for (int i = 0; i < NUM_DOCS; i++){
90 Document doc = new Document();
91 doc.add(new LongField("theLong", theLong--, Field.Store.NO));
92 doc.add(new DoubleField("theDouble", theDouble--, Field.Store.NO));
93 doc.add(new IntField("theInt", theInt--, Field.Store.NO));
94 doc.add(new FloatField("theFloat", theFloat--, Field.Store.NO));
95 if (i%2 == 0) {
96 doc.add(new IntField("sparse", i, Field.Store.NO));
97 }
98
99 if (i%2 == 0) {
100 doc.add(new IntField("numInt", i, Field.Store.NO));
101 }
102
103
104 if (random().nextInt(40) != 17) {
105 unicodeStrings[i] = generateString(i);
106 doc.add(newStringField("theRandomUnicodeString", unicodeStrings[i], Field.Store.YES));
107 }
108
109
110 if (random().nextInt(10) != 8) {
111 for (int j = 0; j < NUM_ORDS; j++) {
112 String newValue = generateString(i);
113 multiValued[i][j] = new BytesRef(newValue);
114 doc.add(newStringField("theRandomUnicodeMultiValuedField", newValue, Field.Store.YES));
115 }
116 Arrays.sort(multiValued[i]);
117 }
118 writer.addDocument(doc);
119 }
120 IndexReader r = writer.getReader();
121 reader = SlowCompositeReaderWrapper.wrap(r);
122 TestUtil.checkReader(reader);
123 writer.close();
124 }
125
126 @AfterClass
127 public static void afterClass() throws Exception {
128 reader.close();
129 reader = null;
130 directory.close();
131 directory = null;
132 unicodeStrings = null;
133 multiValued = null;
134 }
135
136 public void testInfoStream() throws Exception {
137 try {
138 FieldCache cache = FieldCache.DEFAULT;
139 ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
140 cache.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8));
141 cache.getNumerics(reader, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false);
142 cache.getNumerics(reader, "theDouble", new FieldCache.Parser() {
143 @Override
144 public TermsEnum termsEnum(Terms terms) throws IOException {
145 return NumericUtils.filterPrefixCodedLongs(terms.iterator());
146 }
147 @Override
148 public long parseValue(BytesRef term) {
149 int val = (int) NumericUtils.prefixCodedToLong(term);
150 if (val<0) val ^= 0x7fffffff;
151 return val;
152 }
153 }, false);
154 assertTrue(bos.toString(IOUtils.UTF_8).indexOf("WARNING") != -1);
155 } finally {
156 FieldCache.DEFAULT.setInfoStream(null);
157 FieldCache.DEFAULT.purgeAllCaches();
158 }
159 }
160
161 public void test() throws IOException {
162 FieldCache cache = FieldCache.DEFAULT;
163 NumericDocValues doubles = cache.getNumerics(reader, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, random().nextBoolean());
164 assertSame("Second request to cache return same array", doubles, cache.getNumerics(reader, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, random().nextBoolean()));
165 for (int i = 0; i < NUM_DOCS; i++) {
166 assertEquals(Double.doubleToLongBits(Double.MAX_VALUE - i), doubles.get(i));
167 }
168
169 NumericDocValues longs = cache.getNumerics(reader, "theLong", FieldCache.NUMERIC_UTILS_LONG_PARSER, random().nextBoolean());
170 assertSame("Second request to cache return same array", longs, cache.getNumerics(reader, "theLong", FieldCache.NUMERIC_UTILS_LONG_PARSER, random().nextBoolean()));
171 for (int i = 0; i < NUM_DOCS; i++) {
172 assertEquals(Long.MAX_VALUE - i, longs.get(i));
173 }
174
175 NumericDocValues ints = cache.getNumerics(reader, "theInt", FieldCache.NUMERIC_UTILS_INT_PARSER, random().nextBoolean());
176 assertSame("Second request to cache return same array", ints, cache.getNumerics(reader, "theInt", FieldCache.NUMERIC_UTILS_INT_PARSER, random().nextBoolean()));
177 for (int i = 0; i < NUM_DOCS; i++) {
178 assertEquals(Integer.MAX_VALUE - i, ints.get(i));
179 }
180
181 NumericDocValues floats = cache.getNumerics(reader, "theFloat", FieldCache.NUMERIC_UTILS_FLOAT_PARSER, random().nextBoolean());
182 assertSame("Second request to cache return same array", floats, cache.getNumerics(reader, "theFloat", FieldCache.NUMERIC_UTILS_FLOAT_PARSER, random().nextBoolean()));
183 for (int i = 0; i < NUM_DOCS; i++) {
184 assertEquals(Float.floatToIntBits(Float.MAX_VALUE - i), floats.get(i));
185 }
186
187 Bits docsWithField = cache.getDocsWithField(reader, "theLong");
188 assertSame("Second request to cache return same array", docsWithField, cache.getDocsWithField(reader, "theLong"));
189 assertTrue("docsWithField(theLong) must be class Bits.MatchAllBits", docsWithField instanceof Bits.MatchAllBits);
190 assertTrue("docsWithField(theLong) Size: " + docsWithField.length() + " is not: " + NUM_DOCS, docsWithField.length() == NUM_DOCS);
191 for (int i = 0; i < docsWithField.length(); i++) {
192 assertTrue(docsWithField.get(i));
193 }
194
195 docsWithField = cache.getDocsWithField(reader, "sparse");
196 assertSame("Second request to cache return same array", docsWithField, cache.getDocsWithField(reader, "sparse"));
197 assertFalse("docsWithField(sparse) must not be class Bits.MatchAllBits", docsWithField instanceof Bits.MatchAllBits);
198 assertTrue("docsWithField(sparse) Size: " + docsWithField.length() + " is not: " + NUM_DOCS, docsWithField.length() == NUM_DOCS);
199 for (int i = 0; i < docsWithField.length(); i++) {
200 assertEquals(i%2 == 0, docsWithField.get(i));
201 }
202
203
204 SortedDocValues termsIndex = cache.getTermsIndex(reader, "theRandomUnicodeString");
205 for (int i = 0; i < NUM_DOCS; i++) {
206 final String s;
207 final int ord = termsIndex.getOrd(i);
208 if (ord == -1) {
209 s = null;
210 } else {
211 s = termsIndex.lookupOrd(ord).utf8ToString();
212 }
213 assertTrue("for doc " + i + ": " + s + " does not equal: " + unicodeStrings[i], unicodeStrings[i] == null || unicodeStrings[i].equals(s));
214 }
215
216 int nTerms = termsIndex.getValueCount();
217
218 TermsEnum tenum = termsIndex.termsEnum();
219 for (int i=0; i<nTerms; i++) {
220 BytesRef val1 = BytesRef.deepCopyOf(tenum.next());
221 final BytesRef val = termsIndex.lookupOrd(i);
222
223 assertEquals(val, val1);
224 }
225
226
227 int num = atLeast(100);
228 for (int i = 0; i < num; i++) {
229 int k = random().nextInt(nTerms);
230 final BytesRef val = BytesRef.deepCopyOf(termsIndex.lookupOrd(k));
231 assertEquals(TermsEnum.SeekStatus.FOUND, tenum.seekCeil(val));
232 assertEquals(val, tenum.term());
233 }
234
235 for(int i=0;i<nTerms;i++) {
236 final BytesRef val = BytesRef.deepCopyOf(termsIndex.lookupOrd(i));
237 assertEquals(TermsEnum.SeekStatus.FOUND, tenum.seekCeil(val));
238 assertEquals(val, tenum.term());
239 }
240
241
242 termsIndex = cache.getTermsIndex(reader, "bogusfield");
243
244
245 BinaryDocValues terms = cache.getTerms(reader, "theRandomUnicodeString", true);
246 Bits bits = cache.getDocsWithField(reader, "theRandomUnicodeString");
247 for (int i = 0; i < NUM_DOCS; i++) {
248 final String s;
249 if (!bits.get(i)) {
250 s = null;
251 } else {
252 s = terms.get(i).utf8ToString();
253 }
254 assertTrue("for doc " + i + ": " + s + " does not equal: " + unicodeStrings[i], unicodeStrings[i] == null || unicodeStrings[i].equals(s));
255 }
256
257
258 terms = cache.getTerms(reader, "bogusfield", false);
259
260
261 SortedSetDocValues termOrds = cache.getDocTermOrds(reader, "theRandomUnicodeMultiValuedField", null);
262 int numEntries = cache.getCacheEntries().length;
263
264 termOrds = cache.getDocTermOrds(reader, "theRandomUnicodeMultiValuedField", null);
265 assertEquals(numEntries, cache.getCacheEntries().length);
266
267 for (int i = 0; i < NUM_DOCS; i++) {
268 termOrds.setDocument(i);
269
270 List<BytesRef> values = new ArrayList<>(new LinkedHashSet<>(Arrays.asList(multiValued[i])));
271 for (BytesRef v : values) {
272 if (v == null) {
273
274 break;
275 }
276 long ord = termOrds.nextOrd();
277 assert ord != SortedSetDocValues.NO_MORE_ORDS;
278 BytesRef scratch = termOrds.lookupOrd(ord);
279 assertEquals(v, scratch);
280 }
281 assertEquals(SortedSetDocValues.NO_MORE_ORDS, termOrds.nextOrd());
282 }
283
284
285 termOrds = cache.getDocTermOrds(reader, "bogusfield", null);
286 assertTrue(termOrds.getValueCount() == 0);
287
288 FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheKey());
289 }
290
291 public void testEmptyIndex() throws Exception {
292 Directory dir = newDirectory();
293 IndexWriter writer= new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(500));
294 writer.close();
295 IndexReader r = DirectoryReader.open(dir);
296 LeafReader reader = SlowCompositeReaderWrapper.wrap(r);
297 TestUtil.checkReader(reader);
298 FieldCache.DEFAULT.getTerms(reader, "foobar", true);
299 FieldCache.DEFAULT.getTermsIndex(reader, "foobar");
300 FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheKey());
301 r.close();
302 dir.close();
303 }
304
305 private static String generateString(int i) {
306 String s = null;
307 if (i > 0 && random().nextInt(3) == 1) {
308
309 for(int iter = 0; iter < 10 && s == null;iter++) {
310 s = unicodeStrings[random().nextInt(i)];
311 }
312 if (s == null) {
313 s = TestUtil.randomUnicodeString(random());
314 }
315 } else {
316 s = TestUtil.randomUnicodeString(random());
317 }
318 return s;
319 }
320
321 public void testDocsWithField() throws Exception {
322 FieldCache cache = FieldCache.DEFAULT;
323 cache.purgeAllCaches();
324 assertEquals(0, cache.getCacheEntries().length);
325 cache.getNumerics(reader, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, true);
326
327
328
329 assertEquals(2, cache.getCacheEntries().length);
330 Bits bits = cache.getDocsWithField(reader, "theDouble");
331
332
333 assertEquals(2, cache.getCacheEntries().length);
334 assertTrue(bits instanceof Bits.MatchAllBits);
335
336 NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.NUMERIC_UTILS_INT_PARSER, true);
337 assertEquals(4, cache.getCacheEntries().length);
338 Bits docsWithField = cache.getDocsWithField(reader, "sparse");
339 assertEquals(4, cache.getCacheEntries().length);
340 for (int i = 0; i < docsWithField.length(); i++) {
341 if (i%2 == 0) {
342 assertTrue(docsWithField.get(i));
343 assertEquals(i, ints.get(i));
344 } else {
345 assertFalse(docsWithField.get(i));
346 }
347 }
348
349 NumericDocValues numInts = cache.getNumerics(reader, "numInt", FieldCache.NUMERIC_UTILS_INT_PARSER, random().nextBoolean());
350 docsWithField = cache.getDocsWithField(reader, "numInt");
351 for (int i = 0; i < docsWithField.length(); i++) {
352 if (i%2 == 0) {
353 assertTrue(docsWithField.get(i));
354 assertEquals(i, numInts.get(i));
355 } else {
356 assertFalse(docsWithField.get(i));
357 }
358 }
359 }
360
361 public void testGetDocsWithFieldThreadSafety() throws Exception {
362 final FieldCache cache = FieldCache.DEFAULT;
363 cache.purgeAllCaches();
364
365 int NUM_THREADS = 3;
366 Thread[] threads = new Thread[NUM_THREADS];
367 final AtomicBoolean failed = new AtomicBoolean();
368 final AtomicInteger iters = new AtomicInteger();
369 final int NUM_ITER = 200 * RANDOM_MULTIPLIER;
370 final CyclicBarrier restart = new CyclicBarrier(NUM_THREADS,
371 new Runnable() {
372 @Override
373 public void run() {
374 cache.purgeAllCaches();
375 iters.incrementAndGet();
376 }
377 });
378 for(int threadIDX=0;threadIDX<NUM_THREADS;threadIDX++) {
379 threads[threadIDX] = new Thread() {
380 @Override
381 public void run() {
382
383 try {
384 while(!failed.get()) {
385 final int op = random().nextInt(3);
386 if (op == 0) {
387
388
389 restart.await();
390 if (iters.get() >= NUM_ITER) {
391 break;
392 }
393 } else if (op == 1) {
394 Bits docsWithField = cache.getDocsWithField(reader, "sparse");
395 for (int i = 0; i < docsWithField.length(); i++) {
396 assertEquals(i%2 == 0, docsWithField.get(i));
397 }
398 } else {
399 NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.NUMERIC_UTILS_INT_PARSER, true);
400 Bits docsWithField = cache.getDocsWithField(reader, "sparse");
401 for (int i = 0; i < docsWithField.length(); i++) {
402 if (i%2 == 0) {
403 assertTrue(docsWithField.get(i));
404 assertEquals(i, ints.get(i));
405 } else {
406 assertFalse(docsWithField.get(i));
407 }
408 }
409 }
410 }
411 } catch (Throwable t) {
412 failed.set(true);
413 restart.reset();
414 throw new RuntimeException(t);
415 }
416 }
417 };
418 threads[threadIDX].start();
419 }
420
421 for(int threadIDX=0;threadIDX<NUM_THREADS;threadIDX++) {
422 threads[threadIDX].join();
423 }
424 assertFalse(failed.get());
425 }
426
427 public void testDocValuesIntegration() throws Exception {
428 Directory dir = newDirectory();
429 IndexWriterConfig iwc = newIndexWriterConfig(null);
430 RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
431 Document doc = new Document();
432 doc.add(new BinaryDocValuesField("binary", new BytesRef("binary value")));
433 doc.add(new SortedDocValuesField("sorted", new BytesRef("sorted value")));
434 doc.add(new NumericDocValuesField("numeric", 42));
435 doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value1")));
436 doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value2")));
437 iw.addDocument(doc);
438 DirectoryReader ir = iw.getReader();
439 iw.close();
440 LeafReader ar = getOnlySegmentReader(ir);
441
442
443 try {
444 FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.NUMERIC_UTILS_INT_PARSER, false);
445 fail();
446 } catch (IllegalStateException expected) {}
447
448 BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary", true);
449 final BytesRef term = binary.get(0);
450 assertEquals("binary value", term.utf8ToString());
451
452 try {
453 FieldCache.DEFAULT.getTermsIndex(ar, "binary");
454 fail();
455 } catch (IllegalStateException expected) {}
456
457 try {
458 FieldCache.DEFAULT.getDocTermOrds(ar, "binary", null);
459 fail();
460 } catch (IllegalStateException expected) {}
461
462 try {
463 new DocTermOrds(ar, null, "binary");
464 fail();
465 } catch (IllegalStateException expected) {}
466
467 Bits bits = FieldCache.DEFAULT.getDocsWithField(ar, "binary");
468 assertTrue(bits.get(0));
469
470
471 try {
472 FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.NUMERIC_UTILS_INT_PARSER, false);
473 fail();
474 } catch (IllegalStateException expected) {}
475
476 try {
477 new DocTermOrds(ar, null, "sorted");
478 fail();
479 } catch (IllegalStateException expected) {}
480
481 binary = FieldCache.DEFAULT.getTerms(ar, "sorted", true);
482 BytesRef scratch = binary.get(0);
483 assertEquals("sorted value", scratch.utf8ToString());
484
485 SortedDocValues sorted = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
486 assertEquals(0, sorted.getOrd(0));
487 assertEquals(1, sorted.getValueCount());
488 scratch = sorted.get(0);
489 assertEquals("sorted value", scratch.utf8ToString());
490
491 SortedSetDocValues sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sorted", null);
492 sortedSet.setDocument(0);
493 assertEquals(0, sortedSet.nextOrd());
494 assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
495 assertEquals(1, sortedSet.getValueCount());
496
497 bits = FieldCache.DEFAULT.getDocsWithField(ar, "sorted");
498 assertTrue(bits.get(0));
499
500
501 NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.NUMERIC_UTILS_INT_PARSER, false);
502 assertEquals(42, numeric.get(0));
503
504 try {
505 FieldCache.DEFAULT.getTerms(ar, "numeric", true);
506 fail();
507 } catch (IllegalStateException expected) {}
508
509 try {
510 FieldCache.DEFAULT.getTermsIndex(ar, "numeric");
511 fail();
512 } catch (IllegalStateException expected) {}
513
514 try {
515 FieldCache.DEFAULT.getDocTermOrds(ar, "numeric", null);
516 fail();
517 } catch (IllegalStateException expected) {}
518
519 try {
520 new DocTermOrds(ar, null, "numeric");
521 fail();
522 } catch (IllegalStateException expected) {}
523
524 bits = FieldCache.DEFAULT.getDocsWithField(ar, "numeric");
525 assertTrue(bits.get(0));
526
527
528 try {
529 FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.NUMERIC_UTILS_INT_PARSER, false);
530 fail();
531 } catch (IllegalStateException expected) {}
532
533 try {
534 FieldCache.DEFAULT.getTerms(ar, "sortedset", true);
535 fail();
536 } catch (IllegalStateException expected) {}
537
538 try {
539 FieldCache.DEFAULT.getTermsIndex(ar, "sortedset");
540 fail();
541 } catch (IllegalStateException expected) {}
542
543 try {
544 new DocTermOrds(ar, null, "sortedset");
545 fail();
546 } catch (IllegalStateException expected) {}
547
548 sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sortedset", null);
549 sortedSet.setDocument(0);
550 assertEquals(0, sortedSet.nextOrd());
551 assertEquals(1, sortedSet.nextOrd());
552 assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
553 assertEquals(2, sortedSet.getValueCount());
554
555 bits = FieldCache.DEFAULT.getDocsWithField(ar, "sortedset");
556 assertTrue(bits.get(0));
557
558 ir.close();
559 dir.close();
560 }
561
562 public void testNonexistantFields() throws Exception {
563 Directory dir = newDirectory();
564 RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
565 Document doc = new Document();
566 iw.addDocument(doc);
567 DirectoryReader ir = iw.getReader();
568 iw.close();
569
570 LeafReader ar = getOnlySegmentReader(ir);
571
572 final FieldCache cache = FieldCache.DEFAULT;
573 cache.purgeAllCaches();
574 assertEquals(0, cache.getCacheEntries().length);
575
576 NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.NUMERIC_UTILS_INT_PARSER, true);
577 assertEquals(0, ints.get(0));
578
579 NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.NUMERIC_UTILS_LONG_PARSER, true);
580 assertEquals(0, longs.get(0));
581
582 NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.NUMERIC_UTILS_FLOAT_PARSER, true);
583 assertEquals(0, floats.get(0));
584
585 NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, true);
586 assertEquals(0, doubles.get(0));
587
588 BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
589 BytesRef scratch = binaries.get(0);
590 assertEquals(0, scratch.length);
591
592 SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
593 assertEquals(-1, sorted.getOrd(0));
594 scratch = sorted.get(0);
595 assertEquals(0, scratch.length);
596
597 SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued", null);
598 sortedSet.setDocument(0);
599 assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
600
601 Bits bits = cache.getDocsWithField(ar, "bogusbits");
602 assertFalse(bits.get(0));
603
604
605 assertEquals(0, cache.getCacheEntries().length);
606 ir.close();
607 dir.close();
608 }
609
610 public void testNonIndexedFields() throws Exception {
611 Directory dir = newDirectory();
612 RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
613 Document doc = new Document();
614 doc.add(new StoredField("bogusbytes", "bogus"));
615 doc.add(new StoredField("bogusshorts", "bogus"));
616 doc.add(new StoredField("bogusints", "bogus"));
617 doc.add(new StoredField("boguslongs", "bogus"));
618 doc.add(new StoredField("bogusfloats", "bogus"));
619 doc.add(new StoredField("bogusdoubles", "bogus"));
620 doc.add(new StoredField("bogusterms", "bogus"));
621 doc.add(new StoredField("bogustermsindex", "bogus"));
622 doc.add(new StoredField("bogusmultivalued", "bogus"));
623 doc.add(new StoredField("bogusbits", "bogus"));
624 iw.addDocument(doc);
625 DirectoryReader ir = iw.getReader();
626 iw.close();
627
628 LeafReader ar = getOnlySegmentReader(ir);
629
630 final FieldCache cache = FieldCache.DEFAULT;
631 cache.purgeAllCaches();
632 assertEquals(0, cache.getCacheEntries().length);
633
634 NumericDocValues ints = cache.getNumerics(ar, "bogusints", FieldCache.NUMERIC_UTILS_INT_PARSER, true);
635 assertEquals(0, ints.get(0));
636
637 NumericDocValues longs = cache.getNumerics(ar, "boguslongs", FieldCache.NUMERIC_UTILS_LONG_PARSER, true);
638 assertEquals(0, longs.get(0));
639
640 NumericDocValues floats = cache.getNumerics(ar, "bogusfloats", FieldCache.NUMERIC_UTILS_FLOAT_PARSER, true);
641 assertEquals(0, floats.get(0));
642
643 NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, true);
644 assertEquals(0, doubles.get(0));
645
646 BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
647 BytesRef scratch = binaries.get(0);
648 assertEquals(0, scratch.length);
649
650 SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
651 assertEquals(-1, sorted.getOrd(0));
652 scratch = sorted.get(0);
653 assertEquals(0, scratch.length);
654
655 SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued", null);
656 sortedSet.setDocument(0);
657 assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
658
659 Bits bits = cache.getDocsWithField(ar, "bogusbits");
660 assertFalse(bits.get(0));
661
662
663 assertEquals(0, cache.getCacheEntries().length);
664 ir.close();
665 dir.close();
666 }
667
668
669 public void testLongFieldCache() throws IOException {
670 Directory dir = newDirectory();
671 IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random()));
672 cfg.setMergePolicy(newLogMergePolicy());
673 RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg);
674 Document doc = new Document();
675 LongField field = new LongField("f", 0L, Store.YES);
676 doc.add(field);
677 final long[] values = new long[TestUtil.nextInt(random(), 1, 10)];
678 for (int i = 0; i < values.length; ++i) {
679 final long v;
680 switch (random().nextInt(10)) {
681 case 0:
682 v = Long.MIN_VALUE;
683 break;
684 case 1:
685 v = 0;
686 break;
687 case 2:
688 v = Long.MAX_VALUE;
689 break;
690 default:
691 v = TestUtil.nextLong(random(), -10, 10);
692 break;
693 }
694 values[i] = v;
695 if (v == 0 && random().nextBoolean()) {
696
697 iw.addDocument(new Document());
698 } else {
699 field.setLongValue(v);
700 iw.addDocument(doc);
701 }
702 }
703 iw.forceMerge(1);
704 final DirectoryReader reader = iw.getReader();
705 final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.NUMERIC_UTILS_LONG_PARSER, false);
706 for (int i = 0; i < values.length; ++i) {
707 assertEquals(values[i], longs.get(i));
708 }
709 reader.close();
710 iw.close();
711 dir.close();
712 }
713
714
715 public void testIntFieldCache() throws IOException {
716 Directory dir = newDirectory();
717 IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random()));
718 cfg.setMergePolicy(newLogMergePolicy());
719 RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg);
720 Document doc = new Document();
721 IntField field = new IntField("f", 0, Store.YES);
722 doc.add(field);
723 final int[] values = new int[TestUtil.nextInt(random(), 1, 10)];
724 for (int i = 0; i < values.length; ++i) {
725 final int v;
726 switch (random().nextInt(10)) {
727 case 0:
728 v = Integer.MIN_VALUE;
729 break;
730 case 1:
731 v = 0;
732 break;
733 case 2:
734 v = Integer.MAX_VALUE;
735 break;
736 default:
737 v = TestUtil.nextInt(random(), -10, 10);
738 break;
739 }
740 values[i] = v;
741 if (v == 0 && random().nextBoolean()) {
742
743 iw.addDocument(new Document());
744 } else {
745 field.setIntValue(v);
746 iw.addDocument(doc);
747 }
748 }
749 iw.forceMerge(1);
750 final DirectoryReader reader = iw.getReader();
751 final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlySegmentReader(reader), "f", FieldCache.NUMERIC_UTILS_INT_PARSER, false);
752 for (int i = 0; i < values.length; ++i) {
753 assertEquals(values[i], ints.get(i));
754 }
755 reader.close();
756 iw.close();
757 dir.close();
758 }
759
760 }